{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cifar-10-python.tar.gz']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "\n",
    "import time\n",
    "\n",
    "# import pytorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import SGD,Adam,lr_scheduler\n",
    "from torch.utils.data import random_split\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define transformations for train\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(p=.40),\n",
    "    transforms.RandomRotation(30),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define transformations for test\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define training dataloader\n",
    "def get_training_dataloader(train_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        train_transform: transfroms for train dataset\n",
    "        path: path to cifar100 training python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: train_data_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_train = train_transform\n",
    "    cifar10_training = torchvision.datasets.CIFAR10(root='.', train=True, download=True, transform=transform_train)\n",
    "    cifar10_training_loader = DataLoader(\n",
    "        cifar10_training, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_training_loader\n",
    "\n",
    "# define test dataloader\n",
    "def get_testing_dataloader(test_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        test_transform: transforms for test dataset\n",
    "        path: path to cifar100 test python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: cifar100_test_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_test = test_transform\n",
    "    cifar10_test = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)\n",
    "    cifar10_test_loader = DataLoader(\n",
    "        cifar10_test, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement mish activation function\n",
    "def f_mish(input):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "    '''\n",
    "    return input * torch.tanh(F.softplus(input))\n",
    "\n",
    "# implement class wrapper for mish activation function\n",
    "class mish(nn.Module):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = mish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_mish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement swish activation function\n",
    "def f_swish(input):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "    '''\n",
    "    return input * torch.sigmoid(input)\n",
    "\n",
    "# implement class wrapper for swish activation function\n",
    "class swish(nn.Module):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = swish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_swish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BasicResidualSEBlock(nn.Module):\n",
    "\n",
    "    expansion = 1\n",
    "\n",
    "    def __init__(self, in_channels, out_channels, stride, r=16, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            self.activation = F.relu\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            self.activation = f_swish\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "            self.activation = f_mish\n",
    "\n",
    "        self.residual = nn.Sequential(\n",
    "            nn.Conv2d(in_channels, out_channels, 3, stride=stride, padding=1),\n",
    "            nn.BatchNorm2d(out_channels),\n",
    "            f_activation,\n",
    "            \n",
    "            nn.Conv2d(out_channels, out_channels * self.expansion, 3, padding=1),\n",
    "            nn.BatchNorm2d(out_channels * self.expansion),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "        self.shortcut = nn.Sequential()\n",
    "        if stride != 1 or in_channels != out_channels * self.expansion:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                nn.Conv2d(in_channels, out_channels * self.expansion, 1, stride=stride),\n",
    "                nn.BatchNorm2d(out_channels * self.expansion)\n",
    "            )\n",
    "        \n",
    "        self.squeeze = nn.AdaptiveAvgPool2d(1)\n",
    "        self.excitation = nn.Sequential(\n",
    "            nn.Linear(out_channels * self.expansion, out_channels * self.expansion // r),\n",
    "            f_activation,\n",
    "            nn.Linear(out_channels * self.expansion // r, out_channels * self.expansion),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        shortcut = self.shortcut(x)\n",
    "        residual = self.residual(x)\n",
    "\n",
    "        squeeze = self.squeeze(residual)\n",
    "        squeeze = squeeze.view(squeeze.size(0), -1)\n",
    "        excitation = self.excitation(squeeze)\n",
    "        excitation = excitation.view(residual.size(0), residual.size(1), 1, 1)\n",
    "\n",
    "        x = residual * excitation.expand_as(residual) + shortcut\n",
    "\n",
    "        return self.activation(x)\n",
    "\n",
    "class BottleneckResidualSEBlock(nn.Module):\n",
    "\n",
    "    expansion = 4\n",
    "\n",
    "    def __init__(self, in_channels, out_channels, stride, r=16, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            self.activation = F.relu\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            self.activation = f_swish\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "            self.activation = f_mish\n",
    "\n",
    "        self.residual = nn.Sequential(\n",
    "            nn.Conv2d(in_channels, out_channels, 1),\n",
    "            nn.BatchNorm2d(out_channels),\n",
    "            f_activation,\n",
    "\n",
    "            nn.Conv2d(out_channels, out_channels, 3, stride=stride, padding=1),\n",
    "            nn.BatchNorm2d(out_channels),\n",
    "            f_activation,\n",
    "\n",
    "            nn.Conv2d(out_channels, out_channels * self.expansion, 1),\n",
    "            nn.BatchNorm2d(out_channels * self.expansion),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "        self.squeeze = nn.AdaptiveAvgPool2d(1)\n",
    "        self.excitation = nn.Sequential(\n",
    "            nn.Linear(out_channels * self.expansion, out_channels * self.expansion // r),\n",
    "            f_activation,\n",
    "            nn.Linear(out_channels * self.expansion // r, out_channels * self.expansion),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "\n",
    "        self.shortcut = nn.Sequential()\n",
    "        if stride != 1 or in_channels != out_channels * self.expansion:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                nn.Conv2d(in_channels, out_channels * self.expansion, 1, stride=stride),\n",
    "                nn.BatchNorm2d(out_channels * self.expansion)\n",
    "            )\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        shortcut = self.shortcut(x)\n",
    "\n",
    "        residual = self.residual(x)\n",
    "        squeeze = self.squeeze(residual)\n",
    "        squeeze = squeeze.view(squeeze.size(0), -1)\n",
    "        excitation = self.excitation(squeeze)\n",
    "        excitation = excitation.view(residual.size(0), residual.size(1), 1, 1)\n",
    "\n",
    "        x = residual * excitation.expand_as(residual) + shortcut\n",
    "\n",
    "        return self.activation(x)\n",
    "\n",
    "class SEResNet(nn.Module):\n",
    "\n",
    "    def __init__(self, block, block_num, class_num=10, activation = 'relu'):\n",
    "        super().__init__()\n",
    "\n",
    "        self.in_channels = 64\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            self.activation = F.relu\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            self.activation = f_swish\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "            self.activation = f_mish\n",
    "\n",
    "        self.pre = nn.Sequential(\n",
    "            nn.Conv2d(3, 64, 3, padding=1),\n",
    "            nn.BatchNorm2d(64),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "        self.stage1 = self._make_stage(block, block_num[0], 64, 1, activation = activation)\n",
    "        self.stage2 = self._make_stage(block, block_num[1], 128, 2, activation = activation)\n",
    "        self.stage3 = self._make_stage(block, block_num[2], 256, 2, activation = activation)\n",
    "        self.stage4 = self._make_stage(block, block_num[3], 516, 2, activation = activation)\n",
    "\n",
    "        self.linear = nn.Linear(self.in_channels, class_num)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.pre(x)\n",
    "\n",
    "        x = self.stage1(x)\n",
    "        x = self.stage2(x)\n",
    "        x = self.stage3(x)\n",
    "        x = self.stage4(x)\n",
    "\n",
    "        x = F.adaptive_avg_pool2d(x, 1)\n",
    "        x = x.view(x.size(0), -1)\n",
    "\n",
    "        x = self.linear(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "    \n",
    "    def _make_stage(self, block, num, out_channels, stride, activation = 'relu'):\n",
    "\n",
    "        layers = []\n",
    "        layers.append(block(self.in_channels, out_channels, stride, activation = activation))\n",
    "        self.in_channels = out_channels * block.expansion\n",
    "\n",
    "        while num - 1:\n",
    "            layers.append(block(self.in_channels, out_channels, 1, activation = activation))\n",
    "            num -= 1\n",
    "        \n",
    "        return nn.Sequential(*layers)\n",
    "        \n",
    "def seresnet18(activation = 'relu'):\n",
    "    return SEResNet(BasicResidualSEBlock, [2, 2, 2, 2], activation = activation)\n",
    "\n",
    "def seresnet34(activation = 'relu'):\n",
    "    return SEResNet(BasicResidualSEBlock, [3, 4, 6, 3], activation = activation)\n",
    "\n",
    "def seresnet50(activation = 'relu'):\n",
    "    return SEResNet(BottleneckResidualSEBlock, [3, 4, 6, 3], activation = activation)\n",
    "\n",
    "def seresnet101(activation = 'relu'):\n",
    "    return SEResNet(BottleneckResidualSEBlock, [3, 4, 23, 3], activation = activation)\n",
    "\n",
    "def seresnet152(activation = 'relu'):\n",
    "    return SEResNet(BottleneckResidualSEBlock, [3, 8, 36, 3], activation = activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "170500096it [00:06, 27368321.59it/s]                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainloader = get_training_dataloader(train_transform)\n",
    "testloader = get_testing_dataloader(test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 0.001\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = seresnet34(activation = 'mish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# set optimizer, only train the classifier parameters, feature parameters are frozen\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats = pd.DataFrame(columns = ['Epoch', 'Time per epoch', 'Avg time per step', 'Train loss', 'Train accuracy', 'Train top-3 accuracy','Test loss', 'Test accuracy', 'Test top-3 accuracy']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100.. Time per epoch: 99.7130.. Average time per step: 0.2550.. Train loss: 1.5908.. Train accuracy: 0.4104.. Top-3 train accuracy: 0.7458.. Test loss: 1.2833.. Test accuracy: 0.5223.. Top-3 test accuracy: 0.8395\n",
      "Epoch 2/100.. Time per epoch: 99.0360.. Average time per step: 0.2533.. Train loss: 1.1245.. Train accuracy: 0.5945.. Top-3 train accuracy: 0.8761.. Test loss: 0.9677.. Test accuracy: 0.6574.. Top-3 test accuracy: 0.9138\n",
      "Epoch 3/100.. Time per epoch: 98.8100.. Average time per step: 0.2527.. Train loss: 0.9157.. Train accuracy: 0.6730.. Top-3 train accuracy: 0.9126.. Test loss: 0.8330.. Test accuracy: 0.7070.. Top-3 test accuracy: 0.9311\n",
      "Epoch 4/100.. Time per epoch: 98.8087.. Average time per step: 0.2527.. Train loss: 0.7823.. Train accuracy: 0.7266.. Top-3 train accuracy: 0.9316.. Test loss: 0.6411.. Test accuracy: 0.7792.. Top-3 test accuracy: 0.9528\n",
      "Epoch 5/100.. Time per epoch: 98.8049.. Average time per step: 0.2527.. Train loss: 0.6770.. Train accuracy: 0.7614.. Top-3 train accuracy: 0.9477.. Test loss: 0.5678.. Test accuracy: 0.8028.. Top-3 test accuracy: 0.9615\n",
      "Epoch 6/100.. Time per epoch: 98.7964.. Average time per step: 0.2527.. Train loss: 0.5973.. Train accuracy: 0.7924.. Top-3 train accuracy: 0.9576.. Test loss: 0.5259.. Test accuracy: 0.8222.. Top-3 test accuracy: 0.9654\n",
      "Epoch 7/100.. Time per epoch: 98.7789.. Average time per step: 0.2526.. Train loss: 0.5424.. Train accuracy: 0.8110.. Top-3 train accuracy: 0.9635.. Test loss: 0.4923.. Test accuracy: 0.8325.. Top-3 test accuracy: 0.9687\n",
      "Epoch 8/100.. Time per epoch: 98.7725.. Average time per step: 0.2526.. Train loss: 0.4928.. Train accuracy: 0.8285.. Top-3 train accuracy: 0.9676.. Test loss: 0.4561.. Test accuracy: 0.8454.. Top-3 test accuracy: 0.9724\n",
      "Epoch 9/100.. Time per epoch: 98.7578.. Average time per step: 0.2526.. Train loss: 0.4549.. Train accuracy: 0.8414.. Top-3 train accuracy: 0.9721.. Test loss: 0.4609.. Test accuracy: 0.8454.. Top-3 test accuracy: 0.9713\n",
      "Epoch 10/100.. Time per epoch: 98.8028.. Average time per step: 0.2527.. Train loss: 0.4226.. Train accuracy: 0.8527.. Top-3 train accuracy: 0.9757.. Test loss: 0.4361.. Test accuracy: 0.8554.. Top-3 test accuracy: 0.9773\n",
      "Epoch 11/100.. Time per epoch: 98.7779.. Average time per step: 0.2526.. Train loss: 0.3897.. Train accuracy: 0.8636.. Top-3 train accuracy: 0.9779.. Test loss: 0.4253.. Test accuracy: 0.8544.. Top-3 test accuracy: 0.9764\n",
      "Epoch 12/100.. Time per epoch: 98.7317.. Average time per step: 0.2525.. Train loss: 0.3665.. Train accuracy: 0.8713.. Top-3 train accuracy: 0.9812.. Test loss: 0.3986.. Test accuracy: 0.8642.. Top-3 test accuracy: 0.9800\n",
      "Epoch 13/100.. Time per epoch: 98.8777.. Average time per step: 0.2529.. Train loss: 0.3374.. Train accuracy: 0.8812.. Top-3 train accuracy: 0.9837.. Test loss: 0.3842.. Test accuracy: 0.8753.. Top-3 test accuracy: 0.9811\n",
      "Epoch 14/100.. Time per epoch: 98.8010.. Average time per step: 0.2527.. Train loss: 0.3190.. Train accuracy: 0.8879.. Top-3 train accuracy: 0.9842.. Test loss: 0.3866.. Test accuracy: 0.8720.. Top-3 test accuracy: 0.9808\n",
      "Epoch 15/100.. Time per epoch: 98.7853.. Average time per step: 0.2526.. Train loss: 0.2994.. Train accuracy: 0.8949.. Top-3 train accuracy: 0.9867.. Test loss: 0.3882.. Test accuracy: 0.8731.. Top-3 test accuracy: 0.9812\n",
      "Epoch 16/100.. Time per epoch: 98.8349.. Average time per step: 0.2528.. Train loss: 0.2755.. Train accuracy: 0.9054.. Top-3 train accuracy: 0.9877.. Test loss: 0.3681.. Test accuracy: 0.8791.. Top-3 test accuracy: 0.9815\n",
      "Epoch 17/100.. Time per epoch: 98.7842.. Average time per step: 0.2526.. Train loss: 0.2593.. Train accuracy: 0.9089.. Top-3 train accuracy: 0.9893.. Test loss: 0.3653.. Test accuracy: 0.8808.. Top-3 test accuracy: 0.9798\n",
      "Epoch 18/100.. Time per epoch: 98.6742.. Average time per step: 0.2524.. Train loss: 0.2463.. Train accuracy: 0.9142.. Top-3 train accuracy: 0.9905.. Test loss: 0.3568.. Test accuracy: 0.8843.. Top-3 test accuracy: 0.9841\n",
      "Epoch 19/100.. Time per epoch: 98.6939.. Average time per step: 0.2524.. Train loss: 0.2255.. Train accuracy: 0.9202.. Top-3 train accuracy: 0.9925.. Test loss: 0.3635.. Test accuracy: 0.8871.. Top-3 test accuracy: 0.9821\n",
      "Epoch 20/100.. Time per epoch: 98.7075.. Average time per step: 0.2524.. Train loss: 0.2115.. Train accuracy: 0.9247.. Top-3 train accuracy: 0.9927.. Test loss: 0.3576.. Test accuracy: 0.8893.. Top-3 test accuracy: 0.9839\n",
      "Epoch 21/100.. Time per epoch: 98.6268.. Average time per step: 0.2522.. Train loss: 0.2048.. Train accuracy: 0.9275.. Top-3 train accuracy: 0.9934.. Test loss: 0.3541.. Test accuracy: 0.8907.. Top-3 test accuracy: 0.9837\n",
      "Epoch 22/100.. Time per epoch: 98.7503.. Average time per step: 0.2526.. Train loss: 0.1880.. Train accuracy: 0.9340.. Top-3 train accuracy: 0.9943.. Test loss: 0.3951.. Test accuracy: 0.8811.. Top-3 test accuracy: 0.9822\n",
      "Epoch 23/100.. Time per epoch: 99.0113.. Average time per step: 0.2532.. Train loss: 0.1774.. Train accuracy: 0.9378.. Top-3 train accuracy: 0.9946.. Test loss: 0.3657.. Test accuracy: 0.8925.. Top-3 test accuracy: 0.9826\n",
      "Epoch 24/100.. Time per epoch: 98.9052.. Average time per step: 0.2530.. Train loss: 0.1662.. Train accuracy: 0.9410.. Top-3 train accuracy: 0.9952.. Test loss: 0.3758.. Test accuracy: 0.8891.. Top-3 test accuracy: 0.9838\n",
      "Epoch 25/100.. Time per epoch: 98.9200.. Average time per step: 0.2530.. Train loss: 0.1599.. Train accuracy: 0.9441.. Top-3 train accuracy: 0.9957.. Test loss: 0.3692.. Test accuracy: 0.8919.. Top-3 test accuracy: 0.9828\n",
      "Epoch 26/100.. Time per epoch: 99.0103.. Average time per step: 0.2532.. Train loss: 0.1451.. Train accuracy: 0.9487.. Top-3 train accuracy: 0.9963.. Test loss: 0.3936.. Test accuracy: 0.8892.. Top-3 test accuracy: 0.9830\n",
      "Epoch 27/100.. Time per epoch: 98.9833.. Average time per step: 0.2532.. Train loss: 0.1397.. Train accuracy: 0.9507.. Top-3 train accuracy: 0.9967.. Test loss: 0.3862.. Test accuracy: 0.8934.. Top-3 test accuracy: 0.9847\n",
      "Epoch 28/100.. Time per epoch: 99.0795.. Average time per step: 0.2534.. Train loss: 0.1329.. Train accuracy: 0.9527.. Top-3 train accuracy: 0.9971.. Test loss: 0.3599.. Test accuracy: 0.8971.. Top-3 test accuracy: 0.9866\n",
      "Epoch 29/100.. Time per epoch: 99.0853.. Average time per step: 0.2534.. Train loss: 0.1280.. Train accuracy: 0.9554.. Top-3 train accuracy: 0.9971.. Test loss: 0.3771.. Test accuracy: 0.8923.. Top-3 test accuracy: 0.9853\n",
      "Epoch 30/100.. Time per epoch: 99.0581.. Average time per step: 0.2533.. Train loss: 0.1159.. Train accuracy: 0.9588.. Top-3 train accuracy: 0.9978.. Test loss: 0.3924.. Test accuracy: 0.8939.. Top-3 test accuracy: 0.9829\n",
      "Epoch 31/100.. Time per epoch: 99.0523.. Average time per step: 0.2533.. Train loss: 0.1135.. Train accuracy: 0.9610.. Top-3 train accuracy: 0.9978.. Test loss: 0.4060.. Test accuracy: 0.8951.. Top-3 test accuracy: 0.9836\n",
      "Epoch 32/100.. Time per epoch: 98.9826.. Average time per step: 0.2532.. Train loss: 0.1087.. Train accuracy: 0.9621.. Top-3 train accuracy: 0.9981.. Test loss: 0.3714.. Test accuracy: 0.8966.. Top-3 test accuracy: 0.9853\n",
      "Epoch 33/100.. Time per epoch: 98.9574.. Average time per step: 0.2531.. Train loss: 0.0999.. Train accuracy: 0.9656.. Top-3 train accuracy: 0.9982.. Test loss: 0.4048.. Test accuracy: 0.8929.. Top-3 test accuracy: 0.9832\n",
      "Epoch 34/100.. Time per epoch: 99.1295.. Average time per step: 0.2535.. Train loss: 0.0977.. Train accuracy: 0.9661.. Top-3 train accuracy: 0.9983.. Test loss: 0.4009.. Test accuracy: 0.8946.. Top-3 test accuracy: 0.9856\n",
      "Epoch 35/100.. Time per epoch: 99.0464.. Average time per step: 0.2533.. Train loss: 0.0971.. Train accuracy: 0.9657.. Top-3 train accuracy: 0.9984.. Test loss: 0.3997.. Test accuracy: 0.8962.. Top-3 test accuracy: 0.9837\n",
      "Epoch 36/100.. Time per epoch: 99.0294.. Average time per step: 0.2533.. Train loss: 0.0894.. Train accuracy: 0.9688.. Top-3 train accuracy: 0.9982.. Test loss: 0.3883.. Test accuracy: 0.9017.. Top-3 test accuracy: 0.9847\n",
      "Epoch 37/100.. Time per epoch: 99.1260.. Average time per step: 0.2535.. Train loss: 0.0871.. Train accuracy: 0.9697.. Top-3 train accuracy: 0.9986.. Test loss: 0.4098.. Test accuracy: 0.8934.. Top-3 test accuracy: 0.9826\n",
      "Epoch 38/100.. Time per epoch: 99.0599.. Average time per step: 0.2534.. Train loss: 0.0829.. Train accuracy: 0.9712.. Top-3 train accuracy: 0.9990.. Test loss: 0.4191.. Test accuracy: 0.8982.. Top-3 test accuracy: 0.9834\n",
      "Epoch 39/100.. Time per epoch: 98.9784.. Average time per step: 0.2531.. Train loss: 0.0812.. Train accuracy: 0.9718.. Top-3 train accuracy: 0.9989.. Test loss: 0.4052.. Test accuracy: 0.8979.. Top-3 test accuracy: 0.9849\n",
      "Epoch 40/100.. Time per epoch: 99.1613.. Average time per step: 0.2536.. Train loss: 0.0745.. Train accuracy: 0.9740.. Top-3 train accuracy: 0.9987.. Test loss: 0.4712.. Test accuracy: 0.8886.. Top-3 test accuracy: 0.9836\n",
      "Epoch 41/100.. Time per epoch: 98.9923.. Average time per step: 0.2532.. Train loss: 0.0715.. Train accuracy: 0.9750.. Top-3 train accuracy: 0.9991.. Test loss: 0.4411.. Test accuracy: 0.8965.. Top-3 test accuracy: 0.9839\n",
      "Epoch 42/100.. Time per epoch: 99.0042.. Average time per step: 0.2532.. Train loss: 0.0718.. Train accuracy: 0.9748.. Top-3 train accuracy: 0.9991.. Test loss: 0.4298.. Test accuracy: 0.8985.. Top-3 test accuracy: 0.9834\n",
      "Epoch 43/100.. Time per epoch: 99.0370.. Average time per step: 0.2533.. Train loss: 0.0697.. Train accuracy: 0.9758.. Top-3 train accuracy: 0.9991.. Test loss: 0.4297.. Test accuracy: 0.8971.. Top-3 test accuracy: 0.9830\n",
      "Epoch 44/100.. Time per epoch: 99.0977.. Average time per step: 0.2534.. Train loss: 0.0681.. Train accuracy: 0.9753.. Top-3 train accuracy: 0.9992.. Test loss: 0.4442.. Test accuracy: 0.8948.. Top-3 test accuracy: 0.9840\n",
      "Epoch 45/100.. Time per epoch: 99.0342.. Average time per step: 0.2533.. Train loss: 0.0659.. Train accuracy: 0.9777.. Top-3 train accuracy: 0.9991.. Test loss: 0.4534.. Test accuracy: 0.8943.. Top-3 test accuracy: 0.9839\n",
      "Epoch 46/100.. Time per epoch: 99.1457.. Average time per step: 0.2536.. Train loss: 0.0594.. Train accuracy: 0.9793.. Top-3 train accuracy: 0.9994.. Test loss: 0.4471.. Test accuracy: 0.8978.. Top-3 test accuracy: 0.9834\n",
      "Epoch 47/100.. Time per epoch: 99.0886.. Average time per step: 0.2534.. Train loss: 0.0620.. Train accuracy: 0.9783.. Top-3 train accuracy: 0.9993.. Test loss: 0.4556.. Test accuracy: 0.8954.. Top-3 test accuracy: 0.9842\n",
      "Epoch 48/100.. Time per epoch: 99.1450.. Average time per step: 0.2536.. Train loss: 0.0552.. Train accuracy: 0.9806.. Top-3 train accuracy: 0.9993.. Test loss: 0.4364.. Test accuracy: 0.9019.. Top-3 test accuracy: 0.9851\n",
      "Epoch 49/100.. Time per epoch: 99.3064.. Average time per step: 0.2540.. Train loss: 0.0598.. Train accuracy: 0.9793.. Top-3 train accuracy: 0.9993.. Test loss: 0.4566.. Test accuracy: 0.8975.. Top-3 test accuracy: 0.9844\n",
      "Epoch 50/100.. Time per epoch: 99.3681.. Average time per step: 0.2541.. Train loss: 0.0505.. Train accuracy: 0.9823.. Top-3 train accuracy: 0.9995.. Test loss: 0.4415.. Test accuracy: 0.9013.. Top-3 test accuracy: 0.9837\n",
      "Epoch 51/100.. Time per epoch: 99.1652.. Average time per step: 0.2536.. Train loss: 0.0548.. Train accuracy: 0.9809.. Top-3 train accuracy: 0.9994.. Test loss: 0.4373.. Test accuracy: 0.9027.. Top-3 test accuracy: 0.9841\n",
      "Epoch 52/100.. Time per epoch: 99.1405.. Average time per step: 0.2536.. Train loss: 0.0546.. Train accuracy: 0.9809.. Top-3 train accuracy: 0.9995.. Test loss: 0.4326.. Test accuracy: 0.9025.. Top-3 test accuracy: 0.9850\n",
      "Epoch 53/100.. Time per epoch: 99.1503.. Average time per step: 0.2536.. Train loss: 0.0483.. Train accuracy: 0.9833.. Top-3 train accuracy: 0.9996.. Test loss: 0.4327.. Test accuracy: 0.9009.. Top-3 test accuracy: 0.9840\n",
      "Epoch 54/100.. Time per epoch: 99.1830.. Average time per step: 0.2537.. Train loss: 0.0505.. Train accuracy: 0.9826.. Top-3 train accuracy: 0.9995.. Test loss: 0.4286.. Test accuracy: 0.9010.. Top-3 test accuracy: 0.9847\n",
      "Epoch 55/100.. Time per epoch: 99.3698.. Average time per step: 0.2541.. Train loss: 0.0465.. Train accuracy: 0.9837.. Top-3 train accuracy: 0.9996.. Test loss: 0.4533.. Test accuracy: 0.9022.. Top-3 test accuracy: 0.9836\n",
      "Epoch 56/100.. Time per epoch: 99.3819.. Average time per step: 0.2542.. Train loss: 0.0490.. Train accuracy: 0.9830.. Top-3 train accuracy: 0.9995.. Test loss: 0.4443.. Test accuracy: 0.9018.. Top-3 test accuracy: 0.9855\n",
      "Epoch 57/100.. Time per epoch: 99.3213.. Average time per step: 0.2540.. Train loss: 0.0464.. Train accuracy: 0.9841.. Top-3 train accuracy: 0.9995.. Test loss: 0.4486.. Test accuracy: 0.8991.. Top-3 test accuracy: 0.9861\n",
      "Epoch 58/100.. Time per epoch: 99.3426.. Average time per step: 0.2541.. Train loss: 0.0441.. Train accuracy: 0.9840.. Top-3 train accuracy: 0.9996.. Test loss: 0.4593.. Test accuracy: 0.9019.. Top-3 test accuracy: 0.9858\n",
      "Epoch 59/100.. Time per epoch: 99.3034.. Average time per step: 0.2540.. Train loss: 0.0429.. Train accuracy: 0.9850.. Top-3 train accuracy: 0.9995.. Test loss: 0.4492.. Test accuracy: 0.8973.. Top-3 test accuracy: 0.9845\n",
      "Epoch 60/100.. Time per epoch: 99.2744.. Average time per step: 0.2539.. Train loss: 0.0417.. Train accuracy: 0.9860.. Top-3 train accuracy: 0.9995.. Test loss: 0.4626.. Test accuracy: 0.8998.. Top-3 test accuracy: 0.9852\n",
      "Epoch 61/100.. Time per epoch: 99.3615.. Average time per step: 0.2541.. Train loss: 0.0397.. Train accuracy: 0.9861.. Top-3 train accuracy: 0.9998.. Test loss: 0.4434.. Test accuracy: 0.9010.. Top-3 test accuracy: 0.9851\n",
      "Epoch 62/100.. Time per epoch: 99.3949.. Average time per step: 0.2542.. Train loss: 0.0416.. Train accuracy: 0.9847.. Top-3 train accuracy: 0.9996.. Test loss: 0.4497.. Test accuracy: 0.9015.. Top-3 test accuracy: 0.9845\n",
      "Epoch 63/100.. Time per epoch: 99.6054.. Average time per step: 0.2547.. Train loss: 0.0405.. Train accuracy: 0.9856.. Top-3 train accuracy: 0.9996.. Test loss: 0.4724.. Test accuracy: 0.9009.. Top-3 test accuracy: 0.9830\n",
      "Epoch 64/100.. Time per epoch: 99.3100.. Average time per step: 0.2540.. Train loss: 0.0429.. Train accuracy: 0.9852.. Top-3 train accuracy: 0.9997.. Test loss: 0.4540.. Test accuracy: 0.9010.. Top-3 test accuracy: 0.9854\n",
      "Epoch 65/100.. Time per epoch: 99.2580.. Average time per step: 0.2539.. Train loss: 0.0373.. Train accuracy: 0.9874.. Top-3 train accuracy: 0.9997.. Test loss: 0.4507.. Test accuracy: 0.9075.. Top-3 test accuracy: 0.9849\n",
      "Epoch 66/100.. Time per epoch: 99.4331.. Average time per step: 0.2543.. Train loss: 0.0361.. Train accuracy: 0.9876.. Top-3 train accuracy: 0.9998.. Test loss: 0.4774.. Test accuracy: 0.9019.. Top-3 test accuracy: 0.9841\n",
      "Epoch 67/100.. Time per epoch: 99.6251.. Average time per step: 0.2548.. Train loss: 0.0352.. Train accuracy: 0.9877.. Top-3 train accuracy: 0.9998.. Test loss: 0.4783.. Test accuracy: 0.8995.. Top-3 test accuracy: 0.9831\n",
      "Epoch 68/100.. Time per epoch: 99.6682.. Average time per step: 0.2549.. Train loss: 0.0369.. Train accuracy: 0.9877.. Top-3 train accuracy: 0.9997.. Test loss: 0.4814.. Test accuracy: 0.8958.. Top-3 test accuracy: 0.9845\n",
      "Epoch 69/100.. Time per epoch: 99.8521.. Average time per step: 0.2554.. Train loss: 0.0368.. Train accuracy: 0.9870.. Top-3 train accuracy: 0.9997.. Test loss: 0.4758.. Test accuracy: 0.9022.. Top-3 test accuracy: 0.9846\n",
      "Epoch 70/100.. Time per epoch: 99.8921.. Average time per step: 0.2555.. Train loss: 0.0341.. Train accuracy: 0.9883.. Top-3 train accuracy: 0.9997.. Test loss: 0.4712.. Test accuracy: 0.9058.. Top-3 test accuracy: 0.9848\n",
      "Epoch 71/100.. Time per epoch: 99.8040.. Average time per step: 0.2553.. Train loss: 0.0341.. Train accuracy: 0.9882.. Top-3 train accuracy: 0.9996.. Test loss: 0.4777.. Test accuracy: 0.9021.. Top-3 test accuracy: 0.9830\n",
      "Epoch 72/100.. Time per epoch: 99.9996.. Average time per step: 0.2558.. Train loss: 0.0343.. Train accuracy: 0.9877.. Top-3 train accuracy: 0.9997.. Test loss: 0.4641.. Test accuracy: 0.9012.. Top-3 test accuracy: 0.9854\n",
      "Epoch 73/100.. Time per epoch: 99.9967.. Average time per step: 0.2557.. Train loss: 0.0307.. Train accuracy: 0.9897.. Top-3 train accuracy: 0.9998.. Test loss: 0.4713.. Test accuracy: 0.9014.. Top-3 test accuracy: 0.9852\n",
      "Epoch 74/100.. Time per epoch: 99.7994.. Average time per step: 0.2552.. Train loss: 0.0327.. Train accuracy: 0.9889.. Top-3 train accuracy: 0.9997.. Test loss: 0.4668.. Test accuracy: 0.9021.. Top-3 test accuracy: 0.9866\n",
      "Epoch 75/100.. Time per epoch: 99.7350.. Average time per step: 0.2551.. Train loss: 0.0349.. Train accuracy: 0.9874.. Top-3 train accuracy: 0.9998.. Test loss: 0.4616.. Test accuracy: 0.9022.. Top-3 test accuracy: 0.9851\n",
      "Epoch 76/100.. Time per epoch: 99.4250.. Average time per step: 0.2543.. Train loss: 0.0313.. Train accuracy: 0.9896.. Top-3 train accuracy: 0.9998.. Test loss: 0.4870.. Test accuracy: 0.8997.. Top-3 test accuracy: 0.9842\n",
      "Epoch 77/100.. Time per epoch: 99.3819.. Average time per step: 0.2542.. Train loss: 0.0299.. Train accuracy: 0.9897.. Top-3 train accuracy: 0.9998.. Test loss: 0.4844.. Test accuracy: 0.9037.. Top-3 test accuracy: 0.9864\n",
      "Epoch 78/100.. Time per epoch: 99.2523.. Average time per step: 0.2538.. Train loss: 0.0282.. Train accuracy: 0.9901.. Top-3 train accuracy: 0.9998.. Test loss: 0.4786.. Test accuracy: 0.9055.. Top-3 test accuracy: 0.9857\n",
      "Epoch 79/100.. Time per epoch: 99.3179.. Average time per step: 0.2540.. Train loss: 0.0277.. Train accuracy: 0.9903.. Top-3 train accuracy: 0.9997.. Test loss: 0.4960.. Test accuracy: 0.8988.. Top-3 test accuracy: 0.9850\n",
      "Epoch 80/100.. Time per epoch: 99.1639.. Average time per step: 0.2536.. Train loss: 0.0304.. Train accuracy: 0.9899.. Top-3 train accuracy: 0.9998.. Test loss: 0.4920.. Test accuracy: 0.9076.. Top-3 test accuracy: 0.9839\n",
      "Epoch 81/100.. Time per epoch: 99.2267.. Average time per step: 0.2538.. Train loss: 0.0290.. Train accuracy: 0.9897.. Top-3 train accuracy: 0.9999.. Test loss: 0.4849.. Test accuracy: 0.9050.. Top-3 test accuracy: 0.9839\n",
      "Epoch 82/100.. Time per epoch: 99.1526.. Average time per step: 0.2536.. Train loss: 0.0285.. Train accuracy: 0.9906.. Top-3 train accuracy: 0.9998.. Test loss: 0.5099.. Test accuracy: 0.9035.. Top-3 test accuracy: 0.9855\n",
      "Epoch 83/100.. Time per epoch: 99.1649.. Average time per step: 0.2536.. Train loss: 0.0258.. Train accuracy: 0.9908.. Top-3 train accuracy: 0.9998.. Test loss: 0.4940.. Test accuracy: 0.9065.. Top-3 test accuracy: 0.9849\n",
      "Epoch 84/100.. Time per epoch: 99.3085.. Average time per step: 0.2540.. Train loss: 0.0298.. Train accuracy: 0.9893.. Top-3 train accuracy: 0.9998.. Test loss: 0.4830.. Test accuracy: 0.9042.. Top-3 test accuracy: 0.9843\n",
      "Epoch 85/100.. Time per epoch: 99.2509.. Average time per step: 0.2538.. Train loss: 0.0265.. Train accuracy: 0.9912.. Top-3 train accuracy: 0.9998.. Test loss: 0.5099.. Test accuracy: 0.9053.. Top-3 test accuracy: 0.9858\n",
      "Epoch 86/100.. Time per epoch: 99.2115.. Average time per step: 0.2537.. Train loss: 0.0302.. Train accuracy: 0.9902.. Top-3 train accuracy: 0.9999.. Test loss: 0.4809.. Test accuracy: 0.9008.. Top-3 test accuracy: 0.9858\n",
      "Epoch 87/100.. Time per epoch: 99.1899.. Average time per step: 0.2537.. Train loss: 0.0254.. Train accuracy: 0.9911.. Top-3 train accuracy: 0.9998.. Test loss: 0.4677.. Test accuracy: 0.9045.. Top-3 test accuracy: 0.9849\n",
      "Epoch 88/100.. Time per epoch: 99.1388.. Average time per step: 0.2536.. Train loss: 0.0238.. Train accuracy: 0.9916.. Top-3 train accuracy: 0.9999.. Test loss: 0.4951.. Test accuracy: 0.9040.. Top-3 test accuracy: 0.9854\n",
      "Epoch 89/100.. Time per epoch: 99.2081.. Average time per step: 0.2537.. Train loss: 0.0263.. Train accuracy: 0.9910.. Top-3 train accuracy: 0.9998.. Test loss: 0.4876.. Test accuracy: 0.9029.. Top-3 test accuracy: 0.9844\n",
      "Epoch 90/100.. Time per epoch: 99.2278.. Average time per step: 0.2538.. Train loss: 0.0248.. Train accuracy: 0.9913.. Top-3 train accuracy: 0.9999.. Test loss: 0.4949.. Test accuracy: 0.9040.. Top-3 test accuracy: 0.9844\n",
      "Epoch 91/100.. Time per epoch: 99.1975.. Average time per step: 0.2537.. Train loss: 0.0279.. Train accuracy: 0.9904.. Top-3 train accuracy: 0.9998.. Test loss: 0.4944.. Test accuracy: 0.9019.. Top-3 test accuracy: 0.9860\n",
      "Epoch 92/100.. Time per epoch: 99.1976.. Average time per step: 0.2537.. Train loss: 0.0239.. Train accuracy: 0.9917.. Top-3 train accuracy: 0.9999.. Test loss: 0.4885.. Test accuracy: 0.9056.. Top-3 test accuracy: 0.9851\n",
      "Epoch 93/100.. Time per epoch: 99.1737.. Average time per step: 0.2536.. Train loss: 0.0241.. Train accuracy: 0.9921.. Top-3 train accuracy: 0.9999.. Test loss: 0.4872.. Test accuracy: 0.9047.. Top-3 test accuracy: 0.9856\n",
      "Epoch 94/100.. Time per epoch: 99.2005.. Average time per step: 0.2537.. Train loss: 0.0235.. Train accuracy: 0.9922.. Top-3 train accuracy: 0.9999.. Test loss: 0.5046.. Test accuracy: 0.9025.. Top-3 test accuracy: 0.9849\n",
      "Epoch 95/100.. Time per epoch: 99.1982.. Average time per step: 0.2537.. Train loss: 0.0234.. Train accuracy: 0.9919.. Top-3 train accuracy: 0.9998.. Test loss: 0.4789.. Test accuracy: 0.9086.. Top-3 test accuracy: 0.9852\n",
      "Epoch 96/100.. Time per epoch: 99.1726.. Average time per step: 0.2536.. Train loss: 0.0241.. Train accuracy: 0.9917.. Top-3 train accuracy: 0.9999.. Test loss: 0.5042.. Test accuracy: 0.9038.. Top-3 test accuracy: 0.9843\n",
      "Epoch 97/100.. Time per epoch: 99.0922.. Average time per step: 0.2534.. Train loss: 0.0230.. Train accuracy: 0.9921.. Top-3 train accuracy: 0.9999.. Test loss: 0.4946.. Test accuracy: 0.9081.. Top-3 test accuracy: 0.9855\n",
      "Epoch 98/100.. Time per epoch: 99.2597.. Average time per step: 0.2539.. Train loss: 0.0188.. Train accuracy: 0.9934.. Top-3 train accuracy: 0.9999.. Test loss: 0.5016.. Test accuracy: 0.9040.. Top-3 test accuracy: 0.9856\n",
      "Epoch 99/100.. Time per epoch: 99.1056.. Average time per step: 0.2535.. Train loss: 0.0223.. Train accuracy: 0.9920.. Top-3 train accuracy: 0.9999.. Test loss: 0.5213.. Test accuracy: 0.9019.. Top-3 test accuracy: 0.9839\n",
      "Epoch 100/100.. Time per epoch: 99.1277.. Average time per step: 0.2535.. Train loss: 0.0231.. Train accuracy: 0.9924.. Top-3 train accuracy: 0.9999.. Test loss: 0.5152.. Test accuracy: 0.9052.. Top-3 test accuracy: 0.9861\n"
     ]
    }
   ],
   "source": [
    "#train the model\n",
    "model.to(device)\n",
    "\n",
    "steps = 0\n",
    "running_loss = 0\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    since = time.time()\n",
    "    \n",
    "    train_accuracy = 0\n",
    "    top3_train_accuracy = 0 \n",
    "    for inputs, labels in trainloader:\n",
    "        steps += 1\n",
    "        # Move input and label tensors to the default device\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        logps = model.forward(inputs)\n",
    "        loss = criterion(logps, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # calculate train top-1 accuracy\n",
    "        ps = torch.exp(logps)\n",
    "        top_p, top_class = ps.topk(1, dim=1)\n",
    "        equals = top_class == labels.view(*top_class.shape)\n",
    "        train_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "        \n",
    "        # Calculate train top-3 accuracy\n",
    "        np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "        target_numpy = labels.cpu().numpy()\n",
    "        top3_train_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "        \n",
    "    time_elapsed = time.time() - since\n",
    "    \n",
    "    test_loss = 0\n",
    "    test_accuracy = 0\n",
    "    top3_test_accuracy = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in testloader:\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            logps = model.forward(inputs)\n",
    "            batch_loss = criterion(logps, labels)\n",
    "\n",
    "            test_loss += batch_loss.item()\n",
    "\n",
    "            # Calculate test top-1 accuracy\n",
    "            ps = torch.exp(logps)\n",
    "            top_p, top_class = ps.topk(1, dim=1)\n",
    "            equals = top_class == labels.view(*top_class.shape)\n",
    "            test_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "            \n",
    "            # Calculate test top-3 accuracy\n",
    "            np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "            target_numpy = labels.cpu().numpy()\n",
    "            top3_test_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "\n",
    "    print(f\"Epoch {epoch+1}/{epochs}.. \"\n",
    "          f\"Time per epoch: {time_elapsed:.4f}.. \"\n",
    "          f\"Average time per step: {time_elapsed/len(trainloader):.4f}.. \"\n",
    "          f\"Train loss: {running_loss/len(trainloader):.4f}.. \"\n",
    "          f\"Train accuracy: {train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Top-3 train accuracy: {top3_train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Test loss: {test_loss/len(testloader):.4f}.. \"\n",
    "          f\"Test accuracy: {test_accuracy/len(testloader):.4f}.. \"\n",
    "          f\"Top-3 test accuracy: {top3_test_accuracy/len(testloader):.4f}\")\n",
    "\n",
    "    train_stats = train_stats.append({'Epoch': epoch, 'Time per epoch':time_elapsed, 'Avg time per step': time_elapsed/len(trainloader), 'Train loss' : running_loss/len(trainloader), 'Train accuracy': train_accuracy/len(trainloader), 'Train top-3 accuracy':top3_train_accuracy/len(trainloader),'Test loss' : test_loss/len(testloader), 'Test accuracy': test_accuracy/len(testloader), 'Test top-3 accuracy':top3_test_accuracy/len(testloader)}, ignore_index=True)\n",
    "\n",
    "    running_loss = 0\n",
    "    model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats.to_csv('train_log_SENet34_Mish.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
